/*****************************************************************************
 *  ENTROPY - emerging network to reduce orwellian potency yield
 *
 *  Copyright (C) 2002 Juergen Buchmueller <pullmoll@stop1984.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software Foundation,
 *  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 *
 *	$Id: news.c,v 1.4 2005/07/12 23:12:29 pullmoll Exp $
 *****************************************************************************/
#include "news.h"
#include "logger.h"

#define	LINESIZE	4096

#define	INV_ON	"\033[7m"
#define	INV_OFF	"\033[27m"

#define	DIM		"\033[0m"
#define	RED		"\033[0;31m"
#define	GREEN	"\033[0;32m"
#define	BLUE	"\033[0;34m"
#define	HILITE	"\033[0;1m"

news_t *g_news = NULL;
pid_t g_news_pid = (pid_t)-1;

#define	NEWS_LOCK()	osd_sem_wait(&g_news->sem)
#define	NEWS_UNLOCK()	osd_sem_post(&g_news->sem)

int news_sort_board_time(const void *p1, const void *p2)
{
	const news_board_t *b1 = *(const news_board_t **)p1;
	const news_board_t *b2 = *(const news_board_t **)p2;
	if (b2->date.tm_year == b1->date.tm_year) {
		if (b2->date.tm_mon == b1->date.tm_mon) {
			return b2->date.tm_mday - b1->date.tm_mday;
		}
		return b2->date.tm_mon - b1->date.tm_mon;
	}
	return b2->date.tm_year - b1->date.tm_year;
}

int news_sort_entry_time(const void *p1, const void *p2)
{
	const news_entry_t *e1 = *(const news_entry_t **)p1;
	const news_entry_t *e2 = *(const news_entry_t **)p2;
	int delta;
	/* consider time, rounded to days, then message number of that day */
	delta = (e2->time / SECSPERDAY) - (e1->time / SECSPERDAY);
	delta = delta * SECSPERDAY + (e2->slot - e1->slot);
	return delta;
}

/* check if we already have a specific news board */
int news_have_board(chkey_t *key, struct tm *date, const char *board)
{
	news_board_t *b;
	size_t i;
	FUN("news_have_board");

	if (0 != NEWS_LOCK()) {
		LOGS(L_NEWS,L_ERROR,("locking g_news failed (%s)\n",
			strerror(errno)));
		errno = EAGAIN;
		return -1;
	}

	/* scan board list */
	for (i = 0; i < g_news->board_cnt; i++) {
		b = g_news->boardlist[i];
		if (NULL == b)
			continue;
		if (NULL != key) {
			if (0 == memcmp(&key->sha1, &b->key.sha1, SHA1SIZE) &&
				date->tm_year == b->date.tm_year &&
				date->tm_mon == b->date.tm_mon &&
				date->tm_mday == b->date.tm_mday)
				break;
		}
		if (NULL != board) {
			if (0 == strcasecmp(board, b->board) &&
				date->tm_year == b->date.tm_year &&
				date->tm_mon == b->date.tm_mon &&
				date->tm_mday == b->date.tm_mday)
				break;
		}
	}
	if (i < g_news->board_cnt) {
		NEWS_UNLOCK();
		return 0;
	}

	NEWS_UNLOCK();
	errno = ENOENT;
	return -1;
}

/* check if we already have a specific news entry for today */
int news_have_entry(chkey_t *key, struct tm *date)
{
	news_entry_t *e;
	size_t i;
	FUN("news_have_entry");

	if (0 != NEWS_LOCK()) {
		LOGS(L_NEWS,L_ERROR,("locking g_news failed (%s)\n",
			strerror(errno)));
		errno = EAGAIN;
		return -1;
	}

	/* scan entry list */
	for (i = 0; i < g_news->entry_cnt; i++) {
		e = g_news->entrylist[i];
		if (NULL == e)
			continue;
		if (0 == memcmp(&key->sha1, &e->key.sha1, SHA1SIZE) &&
			date->tm_year == e->date.tm_year &&
			date->tm_mon == e->date.tm_mon &&
			date->tm_mday == e->date.tm_mday)
			break;
	}
	if (i < g_news->entry_cnt) {
		NEWS_UNLOCK();
		return 0;
	}

	NEWS_UNLOCK();
	errno = ENOENT;
	return -1;
}

/* add a news board; get board name and allocate memory */
int news_add_board(chkey_t *key, const char *ksk, tempfile_t *tf,
	struct tm *date, size_t slot)
{
	news_board_t *b;
	size_t size, done;
	int rc;
	FUN("news_add_board");

#if	DEBUG == 0
	(void)ksk;
#endif

	if (0 != NEWS_LOCK()) {
		LOGS(L_NEWS,L_ERROR,("locking g_news failed (%s)\n",
			strerror(errno)));
		return -1;
	}

	LOGS(L_NEWS,L_MINOR,("found %s\n", ksk));

	b = (news_board_t *)scalloc(1, sizeof(news_board_t));
	b->key = *key;
	b->slot = slot;
	b->date = *date;

	if (0 != (rc = temp_open(tf))) {
		LOGS(L_NEWS,L_ERROR,("temp_open('%s') call failed (%s)\n",
			tf->name, strerror(errno)));
		NEWS_UNLOCK();
		sfree((caddr_t)b);
		return -1;
	}

	size = tf->wmax;
	if (size > MAX_BOARDSIZE)
		size = MAX_BOARDSIZE;
	if (0 != (rc = temp_read(tf, b->board, size, &done))) {
		NEWS_UNLOCK();
		sfree((caddr_t)b);
		errno = EIO;
		return -1;
	}
	temp_close(tf, 0);

	if (done != size) {
		NEWS_UNLOCK();
		sfree((caddr_t)b);
		errno = EINVAL;
		return -1;
	}

	LOGS(L_NEWS,L_MINOR,("found board '%s' on %s\n", b->board, ksk));

	if (g_news->board_cnt >= g_news->board_max) {
		news_board_t **newlist;
		LOGS(L_NEWS,L_MINOR,("increasing board_max from %u to %u\n",
			(unsigned)(g_news->board_max), (unsigned)(2 * g_news->board_max)));
		g_news->board_max *= 2;
		newlist = (news_board_t **)
			scalloc(g_news->board_max, sizeof(news_board_t *));
		if (NULL == newlist) {
			NEWS_UNLOCK();
			sfree((caddr_t)b);
			errno = ENOMEM;
			return -1;
		}
		memcpy(newlist, g_news->boardlist,
			g_news->board_cnt * sizeof(news_board_t *));
		sfree(g_news->boardlist);
		g_news->boardlist = newlist;
	}

	g_news->boardlist[g_news->board_cnt] = b;
	g_news->board_cnt += 1;
	qsort(g_news->boardlist, g_news->board_cnt, sizeof(news_board_t *),
		news_sort_board_time);

	NEWS_UNLOCK();
	return 0;
}

/* recode a message body in 'buff' from charset 'charset' to utf8 */
int news_utf8_decode(char **pbody, const char *charset, char *buff)
{
	size_t offs, max;
	char *body, *dst, *src;
	const fontmap_t *map = NULL;
	uint32_t ucs = 0;
	FUN("news_utf8_decode");

	*pbody = NULL;

	max = 2 * strlen(buff) + 1;
	body = dst = xcalloc(max, sizeof(char));

	map = wcmapset(charset);

	/* strip off leading UTF-8 escape sequences */
	if (0 == strncmp(buff, "\033%%8", 3) ||
		0 == strncmp(buff, "\033%%G", 3)) {
		buff += 3;
	}

	for (src = buff, dst = body; *src; src++) {
		offs = (size_t)(dst - body);
		if (offs > max - 8) {
			max += 32;
			body = xrealloc(body, max);
			dst = &body[offs];
		}
		if (src[0] == '&' && src[1] == '#' && isdigit(src[2])) {
			offs = 2;
			ucs = 0;
			while (src[offs] != '\0' && src[offs] != ';') {
				ucs = (ucs * 10) + src[offs] - '0';
				offs++;
			}
			if (src[offs] == ';') {
				if (NULL != map && ucs > 0x7f && ucs < 0x100) {
					ucs = wcmapcode(map, (uint8_t)ucs);
					dst += wcencode(dst, ucs, WC_UTF8);
				} else {
					if (ucs < 0x100) {
						*dst++ = ucs;
					} else {
						dst += wcencode(dst, ucs, WC_UTF8);
					}
				}
				src += offs;
			} else {
				*dst++ = *src;
			}
		} else {
			uint8_t ch = (uint8_t)*src;
			if (NULL == map) {
				*dst++ = ch;
			} else {
				if (ch > 0x7f) {
					ucs = wcmapcode(map, ch);
					dst += wcencode(dst, ucs, WC_UTF8);
				} else {
					*dst++ = ch;
				}
			}
		}
	}
	*dst++ = '\0';
	*pbody = body;
	return 0;
}

/* add a news entry; parse the header fields and allocate memory */
int news_add_entry(chkey_t *key, const char *ksk, tempfile_t *tf,
	struct tm *date, size_t slot)
{
	news_entry_t *xe = NULL;
	news_entry_t *se = NULL;
	char *body_orig = NULL;
	char *body_utf8 = NULL;
	char *line = NULL;
	size_t done, size, col, equ, len;
	struct tm tm;
	int f_board = 0;
	int f_from = 0;
	int f_subject = 0;
	int f_charset = 0;
	int f_date = 0;
	int f_time = 0;
	int f_body = 0;
	int eno, rc;
	FUN("news_add_entry");

#if	DEBUG == 0
	(void)ksk;
#endif

	if (0 != NEWS_LOCK()) {
		LOGS(L_NEWS,L_ERROR,("locking g_news failed (%s)\n",
			strerror(errno)));
		return -1;
	}
	LOGS(L_NEWS,L_MINOR,("found %s\n", ksk));

	xe = (news_entry_t *)xcalloc(1, sizeof(news_entry_t));
	xe->key = *key;
	xe->slot = slot;
	xe->date = *date;

	if (0 != (rc = temp_open(tf))) {
		LOGS(L_NEWS,L_ERROR,("temp_open('%s') call failed (%s)\n",
			tf->name, strerror(errno)));
		NEWS_UNLOCK();
		goto bailout;
	}

	line = xcalloc(LINESIZE, sizeof(char));

	size = tf->wmax;
	memset(&tm, 0, sizeof(tm));
	col = equ = 0;
	while (size > 0 && 0 == temp_read(tf, &line[col], 1, NULL)) {
		size--;
		if ('\r' != line[col] && '\n' != line[col]) {
			if (0 == equ && line[col] == '=') {
				equ = col;
			}
			if (col < LINESIZE - 1)
				col++;
			continue;
		}

		line[col] = '\0';
		if (0 == col) {
			continue;
		}
		LOGS(L_NEWS,L_MINOR,("line: '%s'\n", line));

		if (equ > 0) {
			line[equ] = '\0';
			len = col - equ;
			if (0 == strcasecmp(line, "board")) {
				if (col > equ) {
					pm_snprintf(xe->board, sizeof(xe->board), "%s",
						line + equ + 1);
					if (len > sizeof(xe->board))
						len = sizeof(xe->board) - 1;
					f_board = 1;
				} else {
					len = 0;
				}
				xe->board[len] = '\0';
			} else if (0 == strcasecmp(line, "from")) {
				if (col > equ) {
					pm_snprintf(xe->from, sizeof(xe->from), "%s",
						line + equ + 1);
					if (len > sizeof(xe->from))
						len = sizeof(xe->from) - 1;
					f_from = 1;
				} else {
					len = 0;
				}
				xe->from[len] = '\0';
			} else if (0 == strcasecmp(line, "subject")) {
				if (col > equ) {
					pm_snprintf(xe->subject, sizeof(xe->subject), "%s",
						line + equ + 1);
					if (len > sizeof(xe->subject))
						len = sizeof(xe->subject) - 1;
					f_subject = 1;
				} else {
					len = 0;
				}
				xe->subject[len] = '\0';
			} else if (0 == strcasecmp(line, "charset")) {
				if (col > equ) {
					pm_snprintf(xe->charset, sizeof(xe->charset), "%s",
						line + equ + 1);
					if (len > sizeof(xe->charset))
						len = sizeof(xe->charset) - 1;
					f_charset = 1;
				} else {
					len = 0;
				}
				xe->charset[len] = '\0';
			} else if (0 == strcasecmp(line, "date")) {
				if (0 != parse_tm_date(&tm, &line[equ+1])) {
					LOGS(L_NEWS,L_ERROR,("format error date=%s\n",
						line + equ + 1));
				} else {
					xe->time = timegm(&tm);
					f_date = 1;
				}
			} else if (0 == strcasecmp(line, "time")) {
				if (0 != parse_tm_time(&tm, &line[equ+1])) {
					LOGS(L_NEWS,L_ERROR,("format error time=%s\n",
						line + equ + 1));
				} else {
					xe->time = timegm(&tm);
					f_time = 1;
				}
			} else {
				LOGS(L_NEWS,L_MINOR,("unhandled header %s=%s\n",
					line, line + equ + 1));
			}
		} else if (0 == strcasecmp(line, "--- message ---")) {
			/* check the maximum size of a news body */
			if (size > MAX_BODYSIZE) {
				errno = EFBIG;
				rc = -1;
				goto bailout;
			}
			body_orig = xcalloc(size, sizeof(char));
			/* skip the trailing LF of the --- message --- line */
			temp_read(tf, body_orig, 1, NULL);
			size--;
			/* read the remainder as the news body */
			if (0 != temp_read(tf, body_orig, size, &done)) {
				LOGS(L_NEWS,L_ERROR,("temp_read() call failed (%s)\n",
					strerror(errno)));
				errno = EINVAL;
				rc = -1;
				goto bailout;
			}
			if (size != done) {
				LOGS(L_NEWS,L_ERROR,("temp_read() wrong size 0x%x != 0x%x\n",
					(unsigned)size, (unsigned)done));
				errno = EINVAL;
				rc = -1;
				goto bailout;
			}
			rc = news_utf8_decode(&body_utf8, xe->charset, body_orig);
			xfree(body_orig);
			if (0 != rc) {
				LOGS(L_NEWS,L_ERROR,("news_utf8_decode() call failed (%s)\n",
					strerror(errno)));
				errno = EINVAL;
				rc = -1;
				goto bailout;
			}
			pm_snprintf(xe->charset, sizeof(xe->charset), "utf-8");
			size = strlen(body_utf8);
			se = (news_entry_t *)scalloc(1, sizeof(news_entry_t) + size);
			if (NULL == se) {
				LOGS(L_NEWS,L_ERROR,("scalloc(%#x,%#x) call failed (%s)\n",
					1, (unsigned)(sizeof(news_entry_t) + size),
					strerror(errno)));
				errno = ENOMEM;
				rc = -1;
				goto bailout;
			}
			memcpy(se, xe, sizeof(*se));
			strncpy(se->body, body_utf8, size);
			xfree(body_utf8);
			se->body[size] = '\0';
			f_body = 1;
		} else {
			LOGS(L_NEWS,L_ERROR,("line not parsed '%s'\n",
				line));
		}
		col = 0;
		equ = 0;
	}

	temp_close(tf, 0);
	xfree(line);

	/* check if we miss a field */
	if (0 == f_board || 0 == f_from || 0 == f_subject ||
		0 == f_date || 0 == f_time || 0 == f_body) {
		LOGS(L_NEWS,L_ERROR,("format error\n" \
			" board:  %d\n" \
			" from:   %d\n" \
			" subject:%d\n" \
			" charset:%d\n" \
			" date:   %d\n" \
			" time:   %d\n" \
			" body:   %d\n",
			f_board, f_from, f_subject, f_charset, f_date, f_time, f_body));
		errno = EINVAL;
		goto bailout;
	}

	LOGS(L_NEWS,L_MINOR,("found message %s\n" \
		" board:   %s\n" \
		" from:    %s\n" \
		" subject: %s\n" \
		" charset: %s\n" \
		" body:    %d chars\n",
		ksk, se->board, se->from, se->subject, se->charset,
		(int)strlen(se->body)));

	if (g_news->entry_cnt >= g_news->entry_max) {
		news_entry_t **newlist = NULL;
		LOGS(L_NEWS,L_MINOR,("increasing entry_max from %u to %u\n",
			(unsigned)(g_news->entry_max), (unsigned)(2 * g_news->entry_max)));
		g_news->entry_max *= 2;
		newlist = (news_entry_t **)
			scalloc(g_news->entry_max, sizeof(news_entry_t *));
		if (NULL == newlist) {
			NEWS_UNLOCK();
			errno = ENOMEM;
			goto bailout;
		}
		memcpy(newlist, g_news->entrylist,
			g_news->entry_cnt * sizeof(news_entry_t *));
		sfree(g_news->entrylist);
		g_news->entrylist = newlist;
	}
	g_news->entrylist[g_news->entry_cnt] = se;
	g_news->entry_cnt += 1;
	qsort(g_news->entrylist, g_news->entry_cnt, sizeof(news_entry_t *),
		news_sort_entry_time);

	NEWS_UNLOCK();
	xfree(xe);
	xfree(body_utf8);
	xfree(body_orig);
	xfree(line);
	temp_close(tf, 0);
	return 0;

bailout:
	eno = errno;
	NEWS_UNLOCK();
	sfree(se);
	xfree(xe);
	xfree(body_utf8);
	xfree(body_orig);
	xfree(line);
	temp_close(tf, 0);
	errno = eno;
	return rc;
}

/* skip to the next entry to scan, considering retry etc. settings */
int news_skip(int rc)
{
	time_t t0, t1;
	size_t i, j;
	char *comma;
	FUN("news_skip");

	if (0 != NEWS_LOCK()) {
		LOGS(L_NEWS,L_ERROR,("locking g_news failed (%s)\n",
			strerror(errno)));
		return -1;
	}
	/* if the message was found, search the next entry */
	if (0 == rc) {
		g_news->slot += 1;
		NEWS_UNLOCK();
		return 0;
	}

	/* see if we did not yet go through sufficient retries */
	if (g_news->retry[g_news->skip] < g_conf->news_tries) {
		g_news->retry[g_news->skip] += 1;
		g_news->slot += 1;
		g_news->skip += 1;
		if (g_news->skip >= g_conf->news_skip) {
			g_news->slot -= g_conf->news_skip;
			g_news->skip = 0;
		}
		NEWS_UNLOCK();
		return 0;
	}

	/* reset retry counts for this series */
	g_news->slot = 0;
	g_news->skip = 0;
	memset(g_news->retry, 0, sizeof(g_news->retry));

	/* ok, so X consecutive slot failed Y times: try next board */
	g_news->board_idx = g_news->board_idx + 1;
	for (i = 0, j = 1; i < strlen(g_conf->news_boards); i++) {
		if (j >= g_news->board_idx)
			break;
		if (g_conf->news_boards[i] == ',') {
			j++;
		}
	}

	/* if there was no (more) comma, we're back to the first board */
	if (i == strlen(g_conf->news_boards)) {
		g_news->board_idx = 0;
		i = 0;
	}

	/* scanning the _boardlist board? */
	if (0 == g_news->board_idx) {
		strncpy(g_news->board, "_boardlist", sizeof(g_news->board));
		g_news->board[sizeof(g_news->board) - 1] = '\0';
	} else {

		/* copy the board name from position i */
		strncpy(g_news->board, &g_conf->news_boards[i], sizeof(g_news->board));
		g_news->board[sizeof(g_news->board) - 1] = '\0';
		/* remove leading blanks */
		while (g_news->board[0] == ' ')
			strcpy(g_news->board, &g_news->board[1]);
		if (NULL != (comma = strchr(g_news->board, ','))) {
			*comma = '\0';
			/* strip trailing blanks */
			while (comma > g_news->board && *--comma == ' ')
				*comma = '\0';
		}
	}

	LOGS(L_NEWS,L_MINOR,("scanning board '%s'\n",
		g_news->board));

	/* leave now if we're not on the first board again */
	if (0 != g_news->board_idx) {
		NEWS_UNLOCK();
		return 0;
	}

	time(&t0);
	if (1 == (g_news->pass & 1)) {
		/* odd pass no. means scan today */
		memcpy(&g_news->date, gmtime(&t0), sizeof(g_news->date));
		LOGS(L_NEWS,L_MINOR,("scanning date '%04d-%02d-%02d'\n",
			g_news->date.tm_year + 1900,
			g_news->date.tm_mon + 1,
			g_news->date.tm_mday));
		/* increment the scan pass count */
		g_news->pass += 1;
		NEWS_UNLOCK();
		return 0;
	} else {
		/* look back at the past days, decaying from news_days to 1 */
		t1 = SECSPERDAY * (g_conf->news_days - g_news->decay);
		if (g_news->past < t1) {
			g_news->past += SECSPERDAY;
			t0 = t0 - g_news->past;
			memcpy(&g_news->date, gmtime(&t0), sizeof(g_news->date));
			LOGS(L_NEWS,L_MINOR,("scanning date '%04d-%02d-%02d'\n",
				g_news->date.tm_year + 1900,
				g_news->date.tm_mon + 1,
				g_news->date.tm_mday));
			NEWS_UNLOCK();
			return 0;
		}

		/* reset past offset */
		g_news->past = 0;
		/* increase the decay until it reaches news_days - 1 */
		if (g_news->decay + 2 < g_conf->news_days) {
			g_news->decay += 1;
		}

		/* back to today */
		time(&t0);
	}
	memcpy(&g_news->date, gmtime(&t0), sizeof(g_news->date));
	LOGS(L_NEWS,L_MINOR,("scanning date '%04d-%02d-%02d'\n",
		g_news->date.tm_year + 1900,
		g_news->date.tm_mon + 1,
		g_news->date.tm_mday));

	/* increment the scan pass count */
	g_news->pass += 1;

	NEWS_UNLOCK();
	return 0;
}

/* poll the current news slot or _boardlist entry */
int news_poll(int boardlist)
{
	tempfile_t tfdata;
	char *ksk = NULL;
	char *datestr = NULL;
	chkey_t key;
	int htl, rc;
	size_t limit;
	FUN("news_poll");

	temp_invalid(&tfdata);

	pm_asprintf(&datestr, "%d.%d.%d",
		g_news->date.tm_year + 1900,
		g_news->date.tm_mon + 1,
		g_news->date.tm_mday);

	pm_asprintf(&ksk, "KSK@%s/%s/%s-%s-%d.txt",
		g_news->magic,
		g_conf->news_base,
		datestr,
		boardlist ? "_boardlist" : g_news->board,
		g_news->slot);

	if (0 != (rc = create_chk_from_ksk(&key, ksk))) {
		LOGS(L_NEWS,L_ERROR,("create_chk_from_ksk('%s') call failed (%s)\n",
			ksk, strerror(errno)));
		xfree(datestr);
		xfree(ksk);
		return -1;
	}

	/*
	 * reduce htl over time, starting with 1/4th maxhtl
	 */
	htl = g_conf->maxhtl * (g_conf->news_days - g_news->decay) /
		4 / g_conf->news_days;

	/* on pass #0 scan with htl 0 (the local store) */
	if (0 == g_news->pass) {
		htl = 0;
	}

	if (0 != boardlist) {
		if (0 == (rc = news_have_board(&key, &g_news->date, NULL))) {
			LOGS(L_NEWS,L_DEBUG,("already have %s\n", ksk));
			goto bailout;
		}
		limit = MAX_BOARDSIZE;
	} else {
		if (0 != (rc = news_have_board(NULL, &g_news->date, g_news->board))) {
			LOGS(L_NEWS,L_DEBUG,("don't have %s\n", g_news->board));
			goto bailout;
		}
		if (0 == (rc = news_have_entry(&key, &g_news->date))) {
			LOGS(L_NEWS,L_DEBUG,("already have %s\n", ksk));
			goto bailout;
		}
		limit = MAX_BODYSIZE;
	}

	rc = temp_closed(&tfdata, "news-poll", TEMP_UNKNOWN_SIZE);
	if (0 != rc) {
		LOGS(L_NEWS,L_ERROR,("error creating temp file (%s)\n",
			strerror(errno)));
		goto bailout;
	}

	LOGS(L_NEWS,L_MINOR,("searching %s w/ HTL:%d\n", ksk, htl));
	rc = file_get(&key, &tfdata, NULL, htl, &limit, NULL, NULL, 0);
	if (0 != rc) {
		if (EFBIG == errno) {
			LOGS(L_NEWS,L_ERROR,("%s too big (%s)\n",
				ksk, strerror(errno)));
			file_get(&key, NULL, NULL, 0, NULL, NULL, NULL, FILE_ZAP);
			errno = EFBIG;
		}
		goto bailout;
	}

	if (0 != boardlist) {
		rc = news_add_board(&key, ksk, &tfdata,
			&g_news->date, g_news->slot);
	} else {
		rc = news_add_entry(&key, ksk, &tfdata,
			&g_news->date, g_news->slot);
	}
	temp_close(&tfdata, 1);

bailout:
	xfree(datestr);
	xfree(ksk);
	return rc;
}

static int status(news_post_t *np, const char *fmt, ...)
{
	char *buff = NULL;
	size_t length;
	va_list ap;
	FUN("status");

	va_start(ap, fmt);
	length = pm_vasprintf(&buff, fmt, ap);
	va_end(ap);
	if ((size_t)-1 != length && length > 0) {
		(*np->writer)(np->user, buff, length);
	}
	xfree(buff);
	return length;
}

#undef	CHARSET
#define	CHARSET	"iso-8859-1"
int news_post(const char *LANG, news_post_t *np)
{
	tempfile_t tfboard, tfdata;
	char *datestr = NULL;
	char *ksk = NULL;
	char *fquri = NULL;
	chkey_t key, chk_data, chk_board;
	size_t limit, length;
	int htl, try, slot, collision, rc = 0;
	FUN("news_post");

	status(np, "%s:\n",
		LO("posting message"));
	status(np, "%s%s%s\n",
		DIM, np->data, HILITE);
	/* put the board name into a temp file */
	length = strlen(np->board);
	if (0 != (rc = temp_binary(&tfboard, "news-board", length))) {
		LOGS(L_NEWS,L_ERROR,("temp_binary('%s') call failed (%s)\n",
			tfboard.name, strerror(errno)));
		status(np, "%s%s%s\n",
			RED, LO("creating temp board file failed"), HILITE);
		np->rc = -1;
		return -1;
	}
	if (0 != (temp_write(&tfboard, np->board, length))) {
		LOGS(L_NEWS,L_ERROR,("temp_write(%s) call failed (%s)\n",
			tfboard.name, strerror(errno)));
		status(np, "%s%s%s\n",
			RED, LO("writing temp board file failed"), HILITE);
		np->rc = -1;
		temp_close(&tfboard, 1);
		return -1;
	}
	temp_close(&tfboard, 0);

	htl = g_conf->maxhtl / 4 + 1;
	status(np, "%s %s %s %s CHK %s:%d\n",
		LO("inserting board"),
		INV_ON, np->board, INV_OFF,
		LO("with HTL"), htl);
	/* ... and insert as CHK@ w/o meta data */
	rc = file_put(&chk_board, &tfboard, NULL, htl, &collision, NULL, NULL, 0);
	temp_close(&tfboard, 1);
	status(np, "%s %s%s%s\n",
		LO("board key is"), BLUE, key_long(&chk_board), HILITE);

	/* put the message data into a temp file */
	length = strlen(np->data);
	if (0 != (rc = temp_binary(&tfdata, "news-post", length))) {
		LOGS(L_NEWS,L_ERROR,("temp_binary('%s') call failed (%s)\n",
			tfdata.name, strerror(errno)));
		status(np, "%s%s%s\n",
			RED, LO("creating temp message file failed"), HILITE);
		return -1;
	}
	if (0 != (temp_write(&tfdata, np->data, length))) {
		LOGS(L_NEWS,L_ERROR,("temp_write(%s) call failed (%s)\n",
			tfdata.name, strerror(errno)));
		status(np, "%s%s%s\n",
			RED, LO("writing temp message file failed"), HILITE);
		temp_close(&tfdata, 1);
		return -1;
	}
	temp_close(&tfdata, 0);

	htl = g_conf->maxhtl / 4 + 1;
	status(np, "%s CHK %s:%d\n",
		LO("inserting message"), LO("with HTL"), htl);
	/* ... and insert as CHK@ w/o meta data */
	rc = file_put(&chk_data, &tfdata, NULL, htl, &collision, NULL, NULL, 0);
	temp_close(&tfdata, 1);
	status(np, "%s %s%s%s\n",
		LO("message key is"), BLUE, key_long(&chk_data), HILITE);

	/* if we have an error other than a collision */
	if (0 != rc && EEXIST != errno) {
		LOGS(L_NEWS,L_ERROR,("file_put() call failed (%s)\n",
			strerror(errno)));
		rc = -1;
		goto bailout;
	}

	pm_asprintf(&datestr, "%d.%d.%d",
		np->date.tm_year + 1900,
		np->date.tm_mon + 1,
		np->date.tm_mday);

	if (0 != news_have_board(NULL, &np->date, np->board)) {
		status(np, "%s %s %s %s %s %04d-%02d-%02d\n",
			LO("did not find board"),
			INV_ON, np->board, INV_OFF,
			LO("for"),
			np->date.tm_year + 1900,
			np->date.tm_mon + 1,
			np->date.tm_mday);
		for (slot = 0; /* */; slot++) {
			xfree(ksk);
			pm_asprintf(&ksk, "KSK@%s/%s/%s-%s-%d.txt",
				g_news->magic,
				g_conf->news_base,
				datestr,
				"_boardlist",
				slot);

			/* create the CHK for this key */
			if (0 != (rc = create_chk_from_ksk(&key, ksk))) {
				LOGS(L_NEWS,L_ERROR,("create_chk_from_ksk('%s') failed (%s)\n",
					ksk, strerror(errno)));
				goto bailout;
			}

			for (htl = 2; htl <= 16; htl *= 2) {
				status(np, "%s %s%s%s %s:%d\r",
					LO("searching"), GREEN, ksk, HILITE, LO("with HTL"), htl);
				/* then try to fetch it with htl=x now ... */
				rc = temp_closed(&tfdata, "news-poll", TEMP_UNKNOWN_SIZE);
				limit = MAX_BOARDSIZE;
				rc = file_get(&key, &tfdata, NULL, htl, &limit, NULL, NULL, 0);
				if (0 == rc) {
					status(np, "\n%s %s%s%s\n",
						LO("just found board"), GREEN, ksk, HILITE);
					LOGS(L_NEWS,L_DEBUG,("just found %s\n", ksk));
		 			news_add_board(&key, ksk, &tfdata, &np->date, slot);
				}
				temp_close(&tfdata, 1);
				if (0 == rc)
					break;
			}

			if (0 == news_have_board(NULL, &np->date, np->board)) {
				status(np, "%s %s %s %s\n",
					LO("no need to insert"), INV_ON, np->board, INV_OFF);
				break;
			}

			/* found a board slot, continue to search for an empty slot */
			if (0 == rc)
				continue;

			/* okay, let's try to insert the KSK redirect */
			htl = g_conf->maxhtl / 4 + 1;
			status(np, "%s %s%s%s %s:%d\n",
				LO("inserting"), GREEN, ksk, HILITE, LO("with HTL"), htl);
			for (try = 1; try < 8; try++) {
				rc = file_put_redirect(&key, &chk_board, ksk, &fquri,
					htl, &collision, 0);
				if (0 == rc)
					break;
			}

			/* if there was no error */
			if (0 == rc) {
				status(np, "%s %s%s%s\n",
					LO("successfully inserted"), GREEN, ksk, HILITE);
				LOGS(L_NEWS,L_DEBUG,("successfully inserted %s at %s\n",
					fquri, key_short(&key)));
				break;
			} else if (EEXIST == errno) {
				LOGS(L_NEWS,L_DEBUG,("found %s while trying to insert it\n",
					ksk));
				status(np, "%s %s%s%s\n",
					LO("just found board"), GREEN, ksk, HILITE);
				continue;
			} else {
				LOGS(L_NEWS,L_ERROR,("failed to insert redirect for %s to %s (%s)\n",
					ksk, key_short(&chk_data), strerror(errno)));
				status(np, "%s %s%s%s\n",
					LO("failed to insert redirect for"), GREEN, ksk, HILITE);
				goto bailout;
			}
		}
	}

	if (0 != news_have_entry(&chk_data, &np->date)) {
		LOGS(L_NEWS,L_NORMAL,("did not find msg %s for %04d-%02d-%02d\n",
			key_long(&chk_data),
			np->date.tm_year + 1900,
			np->date.tm_mon + 1,
			np->date.tm_mday));
		for (slot = 0; /* */; slot++) {
			xfree(ksk);
			pm_asprintf(&ksk, "KSK@%s/%s/%s-%s-%d.txt",
				g_news->magic,
				g_conf->news_base,
				datestr,
				np->board,
				slot);

			/* create the CHK for this key */
			if (0 != (rc = create_chk_from_ksk(&key, ksk))) {
				LOGS(L_NEWS,L_ERROR,("create_chk_from_ksk('%s') failed (%s)\n",
					ksk, strerror(errno)));
				goto bailout;
			}

			/* check if it was already found */
			if (0 == news_have_entry(&key, &np->date)) {
				LOGS(L_NEWS,L_DEBUG,("already have %s\n", ksk));
				continue;
			}

			for (htl = 2; htl <= 16; htl *= 2) {
				status(np, "%s %s%s%s %s:%d\r",
					LO("searching"), GREEN, ksk, HILITE, LO("with HTL"), htl);
				/* then try to fetch it with maxhtl now ... */
				rc = temp_closed(&tfdata, "news-poll", TEMP_UNKNOWN_SIZE);
				limit = MAX_BODYSIZE;
				rc = file_get(&key, &tfdata, NULL, htl, &limit, NULL, NULL, 0);
				if (0 == rc) {
					status(np, "\n%s %s%s%s\n",
						LO("just found message"), GREEN, ksk, HILITE);
					LOGS(L_NEWS,L_DEBUG,("just found %s\n", ksk));
					news_add_entry(&key, ksk, &tfdata, &np->date, slot);
				}
				temp_close(&tfdata, 1);
				if (0 == rc)
					break;
			}

			/* found a message slot, continue to search empty slot */
			if (0 == rc)
				continue;

			/* okay, let's try to insert the KSK redirect */
			htl = g_conf->maxhtl / 4 + 1;
			status(np, "%s %s%s%s %s:%d\n",
				LO("inserting slot"), GREEN, ksk, HILITE, LO("with HTL"), htl);
			for (try = 1; try < 8; try++) {
				rc = file_put_redirect(&key, &chk_data, ksk, &fquri,
					htl, &collision, 0);
				if (0 == rc)
					break;
			}

			/* if there was no error */
			if (0 == rc) {
				status(np, "%s %s%s%s\n",
					LO("successfully inserted message"), GREEN, ksk, HILITE);
				LOGS(L_NEWS,L_NORMAL,("successfully inserted %s at %s\n",
					fquri, key_short(&key)));
				break;
			} else if (EEXIST == errno) {
				status(np, "%s %s%s%s\n",
					LO("just found message"), GREEN, ksk, HILITE);
				LOGS(L_NEWS,L_DEBUG,("found %s while trying to insert it\n", ksk));
				continue;
			} else {
				status(np, "%s %s%s%s\n",
					LO("failed to insert redirect for"), GREEN, ksk, HILITE);
				LOGS(L_NEWS,L_ERROR,("failed to insert redirect for %s to %s (%s)\n",
					ksk, key_short(&chk_data), strerror(errno)));
				goto bailout;
			}
		}
	}

bailout:
	temp_close(&tfboard, 1);
	temp_close(&tfdata, 1);
	if (0 == rc) {
		status(np, "%s %s%s%s\n",
			LO("success"), GREEN, fquri, HILITE);
		strncpy(np->fquri, fquri, sizeof(np->fquri));
		np->rc = 0;
	} else {
		status(np, "%s%s%s\n",
			RED, LO("failed to insert message"), HILITE);
		np->rc = -1;
	}
	xfree(datestr);
	xfree(fquri);
	xfree(ksk);
	return rc;
}

/* shut down the news polling daemon */
int news_close(void)
{
	size_t i;
	int rc = 0;
	FUN("news_close");

	if (NULL != g_news) {
		for (i = 0; i < g_news->board_cnt; i++)
			sfree((caddr_t)g_news->boardlist[i]);
		for (i = 0; i < g_news->entry_cnt; i++)
			sfree((caddr_t)g_news->entrylist[i]);
		sfree((caddr_t)g_news);
		g_news = NULL;
	}

	return rc;
}

static void news_exit(int sig)
{
	pid_t pid = getpid();
	FUN("news_exit");

	signal(sig, SIG_DFL);
	LOGS(L_NEWS,L_MINOR,("*** {%d} signal %s ***\n",
		(int)pid, signal_name(sig)));
	if (pid == g_news_pid) {
		g_news_pid = (pid_t)-1;
	}
}

int news(void)
{
	size_t size, delay;
	time_t t0;
	int rc = 0;
	FUN("news");

	size = sizeof(news_t);
	g_news = (news_t *)scalloc(size, 1);
	if (NULL == g_news) {
		LOGS(L_NEWS,L_ERROR,("failed to scalloc() news (size %d) (%s)\n",
			(int)size, strerror(errno)));
		return -1;
	}
	/* set PID of g_news shm to zero so the forked child doesn't remove it */
	spset((caddr_t)g_news, 0);

	g_news->board_max = MAX_BOARDS;
	g_news->boardlist = (news_board_t **)
		scalloc(g_news->board_max, sizeof(news_board_t *));
	g_news->entry_max = MAX_NEWS;
	g_news->entrylist = (news_entry_t **)
		scalloc(g_news->entry_max, sizeof(news_entry_t *));

	if (0 != (rc = osd_sem_init(&g_news->sem, 1, 1))) {
		LOGS(L_NEWS,L_ERROR,("osd_sem_init(%p,%d,%d) call failed (%s)\n",
			&g_news->sem, 1, 1, strerror(errno)));
		return -1;
	}

	strncpy(g_news->magic, "frost/message", sizeof(g_news->magic));
	if (g_conf->news_days < 1) {
		LOGS(L_NEWS,L_NORMAL,("raised news_days to %d (was %d)\n",
			1, (int)g_conf->news_days));
		g_conf->news_days = 1;
	}
	size = sizeof(g_news->retry) / sizeof(g_news->retry[0]);
	if (g_conf->news_tries > size) {
		LOGS(L_NEWS,L_NORMAL,("clipped news_retries to %d (was %d)\n",
			(int)size, (int)g_conf->news_tries));
		g_conf->news_tries = size;
	}
	time(&t0);
	memcpy(&g_news->date, gmtime(&t0), sizeof(g_news->date));

	switch (osd_fork2("news", g_conf->niceness, -1)) {
	case -1:
		LOGS(L_NEWS,L_ERROR,("osd_fork2('%s',%d) failed (%s)\n",
			"news", g_conf->niceness, strerror(errno)));
		osd_exit(errno);
	case 0:
		g_news_pid = getpid();
		/* capture signals */
		set_signal_handler(SIGHUP, news_exit);
		set_signal_handler(SIGINT, news_exit);
		set_signal_handler(SIGPIPE, news_exit);
		set_signal_handler(SIGALRM, news_exit);
		set_signal_handler(SIGTERM, news_exit);

		/* sleep 30 seconds before we run into the loop scanning for news */
		osd_sleep(30);

		for (;;) {
			if ((pid_t)-1 == g_news_pid ||
				(pid_t)0 == g_news_pid) {
				LOGS(L_NEWS,L_DEBUG,("g_news_pid is invalid - breaking out\n"));
				break;
			}
			if (0 == strlen(g_conf->news_boards)) {
				osd_sleep(15);
				continue;
			}
			if (0 != g_news->rescan) {
				g_news->rescan = 0;
				g_news->pass = 0;
				g_news->past = 0;
				g_news->decay = 0;
				g_news->slot = 0;
				g_news->skip = 0;
				memset(g_news->retry, 0, sizeof(g_news->retry));
				g_news->board_idx = 0;
			}
			/* if we reached board index 0, scan the _boardlist */
			if (0 == g_news->board_idx) {
				rc = news_poll(1);
			} else {
				rc = news_poll(0);
			}
			news_skip(rc);

			/* 0.5s, 1s, 1.5s and increasing up 10s per message poll */
			if (g_news->pass < 20) {
				delay = 500 * (1 + g_news->pass);
			} else {
				delay = 10000;
			} 
			osd_usleep(delay * 1000);
		}
		news_close();
		osd_exit(rc);
	}

	return rc;
}
